{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.sparse import csr_matrix\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "电影数目（有名称）：33258\n",
      "电影数目（没有名称）：24166\n",
      "电影数目（总计）：57424\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>43061</th>\n",
       "      <td>43061</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28477</th>\n",
       "      <td>28477</td>\n",
       "      <td>Du er ikke alene</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46119</th>\n",
       "      <td>46119</td>\n",
       "      <td>大明奇才</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52907</th>\n",
       "      <td>52907</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34785</th>\n",
       "      <td>34785</td>\n",
       "      <td>ハンチョウ〜神南署安積班〜</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36043</th>\n",
       "      <td>36043</td>\n",
       "      <td>Plus Belle La Vie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54048</th>\n",
       "      <td>54048</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55479</th>\n",
       "      <td>55479</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8137</th>\n",
       "      <td>8137</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39433</th>\n",
       "      <td>39433</td>\n",
       "      <td>Amor Só de Mãe</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       movieId              title\n",
       "43061    43061                NaN\n",
       "28477    28477   Du er ikke alene\n",
       "46119    46119               大明奇才\n",
       "52907    52907                NaN\n",
       "34785    34785      ハンチョウ〜神南署安積班〜\n",
       "36043    36043  Plus Belle La Vie\n",
       "54048    54048                NaN\n",
       "55479    55479                NaN\n",
       "8137      8137                NaN\n",
       "39433    39433     Amor Só de Mãe"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies = pd.read_csv( './data/douban/movies.csv')\n",
    "\n",
    "print('电影数目（有名称）：%d' % movies[~pd.isnull(movies.title)].shape[0])\n",
    "print('电影数目（没有名称）：%d' % movies[pd.isnull(movies.title)].shape[0])\n",
    "print('电影数目（总计）：%d' % movies.shape[0])\n",
    "movies.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "用户数据：28718\n",
      "电影数据：57424\n",
      "评分数目：2828500\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1318222486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1313813583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1313458035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>1313327802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1312126734</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating   timestamp\n",
       "0       0        0       5  1318222486\n",
       "1       0        1       4  1313813583\n",
       "2       0        2       5  1313458035\n",
       "3       0        3       5  1313327802\n",
       "4       0        4       3  1312126734"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings = pd.read_csv('./data/douban/ratings.csv')\n",
    "print('用户数据：%d' % ratings.userId.unique().shape[0])\n",
    "print('电影数据：%d' % ratings.movieId.unique().shape[0])\n",
    "print('评分数目：%d' % ratings.shape[0])\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2828500\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>529</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1247</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1335</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1397</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating title\n",
       "0       0        0       5   NaN\n",
       "1     529        0       4   NaN\n",
       "2    1247        0       5   NaN\n",
       "3    1335        0       5   NaN\n",
       "4    1397        0       5   NaN"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_movie_rating= pd.merge(ratings,movies,on='movieId')\n",
    "combine_movie_rating=combine_movie_rating.drop(['timestamp'],axis = 1)\n",
    "print(len(combine_movie_rating))\n",
    "combine_movie_rating.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2604995\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    userId  movieId  rating                                          title\n",
       "22       0        1       4  Harry Potter and the Deathly Hallows: Part II\n",
       "23      21        1       4  Harry Potter and the Deathly Hallows: Part II\n",
       "24      25        1       5  Harry Potter and the Deathly Hallows: Part II\n",
       "25      34        1       4  Harry Potter and the Deathly Hallows: Part II\n",
       "26      36        1       5  Harry Potter and the Deathly Hallows: Part II"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combine_movie_rating = combine_movie_rating.dropna(axis = 0 ,subset=['title'])\n",
    "print(len(combine_movie_rating))\n",
    "combine_movie_rating.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>totalRatingCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>1898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>2218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>4981</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId  totalRatingCount\n",
       "0        1              1703\n",
       "1        2              1080\n",
       "2        4              1898\n",
       "3        5              2218\n",
       "4       10              4981"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_rating_count=pd.DataFrame(combine_movie_rating.\n",
    "                    groupby(['movieId'])['rating'].\n",
    "                    count().\n",
    "                    reset_index().\n",
    "                    rename(columns={'rating':'totalRatingCount'})                   \n",
    "                   )\n",
    "movie_rating_count.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count   33258.000\n",
      "mean       78.327\n",
      "std       262.606\n",
      "min         1.000\n",
      "25%         3.000\n",
      "50%        10.000\n",
      "75%        38.000\n",
      "max      6574.000\n",
      "Name: totalRatingCount, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "pd.set_option('display.float_format', lambda x: '%.3f' % x)\n",
    "print(movie_rating_count['totalRatingCount'].describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.900    158.000\n",
      "0.910    184.000\n",
      "0.920    211.440\n",
      "0.930    253.000\n",
      "0.940    303.580\n",
      "0.950    375.150\n",
      "0.960    462.000\n",
      "0.970    590.000\n",
      "0.980    814.860\n",
      "0.990   1298.860\n",
      "Name: totalRatingCount, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(movie_rating_count['totalRatingCount'].quantile(np.arange(.9,1,.01)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2604995\n",
      "2604995\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>title</th>\n",
       "      <th>totalRatingCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating                                          title  \\\n",
       "0       0        1       4  Harry Potter and the Deathly Hallows: Part II   \n",
       "1      21        1       4  Harry Potter and the Deathly Hallows: Part II   \n",
       "2      25        1       5  Harry Potter and the Deathly Hallows: Part II   \n",
       "3      34        1       4  Harry Potter and the Deathly Hallows: Part II   \n",
       "4      36        1       5  Harry Potter and the Deathly Hallows: Part II   \n",
       "\n",
       "   totalRatingCount  \n",
       "0              1703  \n",
       "1              1703  \n",
       "2              1703  \n",
       "3              1703  \n",
       "4              1703  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_with_totalRatingCount = combine_movie_rating.merge(movie_rating_count,left_on='movieId',right_on='movieId')\n",
    "print(len(combine_movie_rating))\n",
    "print(len(rating_with_totalRatingCount))\n",
    "rating_with_totalRatingCount.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>title</th>\n",
       "      <th>totalRatingCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Harry Potter and the Deathly Hallows: Part II</td>\n",
       "      <td>1703</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating                                          title  \\\n",
       "0       0        1       4  Harry Potter and the Deathly Hallows: Part II   \n",
       "1      21        1       4  Harry Potter and the Deathly Hallows: Part II   \n",
       "2      25        1       5  Harry Potter and the Deathly Hallows: Part II   \n",
       "3      34        1       4  Harry Potter and the Deathly Hallows: Part II   \n",
       "4      36        1       5  Harry Potter and the Deathly Hallows: Part II   \n",
       "\n",
       "   totalRatingCount  \n",
       "0              1703  \n",
       "1              1703  \n",
       "2              1703  \n",
       "3              1703  \n",
       "4              1703  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#有10%的电影评价次数大于158次\n",
    "popular_threshold=158\n",
    "rating_popular_movies= rating_with_totalRatingCount.query('totalRatingCount>=@popular_threshold')\n",
    "rating_popular_movies.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_pivot = rating_popular_movies.pivot(index='movieId', columns='userId',values='rating').fillna(0)\n",
    "ratings_pivot_sparse = csr_matrix(ratings_pivot.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NearestNeighbors(algorithm='brute', leaf_size=30, metric='cosine',\n",
       "         metric_params=None, n_jobs=1, n_neighbors=5, p=2, radius=1.0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_nn_binary = NearestNeighbors(metric='cosine', algorithm='brute')\n",
    "model_nn_binary.fit(ratings_pivot_sparse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "当前电影: 少年张三丰\n",
      "推荐电影1: 机灵小不懂, 距离为:0.4828775024976728\n",
      "推荐电影2: 春光灿烂猪八戒, 距离为:0.5471577246480848\n",
      "推荐电影3: 绝代双骄, 距离为:0.5736253341325162\n",
      "推荐电影4: 粉红女郎, 距离为:0.5761728959194434\n",
      "推荐电影5: 少年包青天, 距离为:0.578347365754289\n",
      "推荐电影6: 少年英雄方世玉, 距离为:0.5805705904995269\n",
      "推荐电影7: 少年包青天2, 距离为:0.5844965129491082\n",
      "推荐电影8: 小李飞刀, 距离为:0.5905323242444609\n",
      "推荐电影9: 穿越时空的爱恋, 距离为:0.5968249361452425\n",
      "推荐电影10: 還珠格格第二部, 距离为:0.6047020203715006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:2: FutureWarning: reshape is deprecated and will raise in a subsequent release. Please use .values.reshape(...) instead\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "query_index=np.random.choice(ratings_pivot.shape[0])\n",
    "distances, indices = model_nn_binary.kneighbors(ratings_pivot.iloc[query_index, :].reshape(1, -1), n_neighbors = 11)\n",
    "\n",
    "for i in range(0, len(distances.flatten())):\n",
    "    movieId=ratings_pivot.index[query_index]\n",
    "    likelymovieId=ratings_pivot.index[indices.flatten()[i]]\n",
    "    if i == 0:\n",
    "        print('当前电影:',movies[movies.movieId==movieId]['title'].values[0])\n",
    "    else:\n",
    "        print('推荐电影{0}: {1}, 距离为:{2}'.format(i, movies[movies.movieId==likelymovieId]['title'].values[0], \n",
    "                                                    distances.flatten()[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "当前电影: 黃飛鴻之三獅王爭霸\n",
      "推荐电影1: 黃飛鴻之二男兒當自強, 距离为:0.2236617772379993\n",
      "推荐电影2: 黃飛鴻, 距离为:0.2771905809331011\n",
      "推荐电影3: 方世玉, 距离为:0.3036596141407937\n",
      "推荐电影4: 太极张三丰, 距离为:0.3454806002742725\n",
      "推荐电影5: 方世玉续集, 距离为:0.3489783330675462\n",
      "推荐电影6: 精武英雄, 距离为:0.38593751478880156\n",
      "推荐电影7: 新少林五祖, 距离为:0.4035055605033421\n",
      "推荐电影8: 倚天屠龍記之魔教教主, 距离为:0.4150684119775777\n",
      "推荐电影9: 中南海保镖, 距离为:0.45784800244154567\n",
      "推荐电影10: 我是谁, 距离为:0.45895083257042657\n"
     ]
    }
   ],
   "source": [
    "movieId=2550\n",
    "distances, indices = model_nn_binary.kneighbors(ratings_pivot.query('movieId == 2550').values, n_neighbors = 11)\n",
    "\n",
    "for i in range(0, len(distances.flatten())):\n",
    "    likelymovieId=ratings_pivot.index[indices.flatten()[i]]\n",
    "    if i == 0:\n",
    "        print('当前电影:',movies[movies.movieId==movieId]['title'].values[0])\n",
    "    else:\n",
    "        print('推荐电影{0}: {1}, 距离为:{2}'.format(i, movies[movies.movieId==likelymovieId]['title'].values[0], \n",
    "                                                    distances.flatten()[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(27895, 3329)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>movieId</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>15</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>...</th>\n",
       "      <th>12612</th>\n",
       "      <th>12634</th>\n",
       "      <th>13346</th>\n",
       "      <th>14821</th>\n",
       "      <th>15721</th>\n",
       "      <th>15741</th>\n",
       "      <th>15826</th>\n",
       "      <th>16155</th>\n",
       "      <th>16323</th>\n",
       "      <th>16660</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>userId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.000</td>\n",
       "      <td>5.000</td>\n",
       "      <td>3.000</td>\n",
       "      <td>4.000</td>\n",
       "      <td>5.000</td>\n",
       "      <td>4.000</td>\n",
       "      <td>2.000</td>\n",
       "      <td>4.000</td>\n",
       "      <td>2.000</td>\n",
       "      <td>2.000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>3.000</td>\n",
       "      <td>4.000</td>\n",
       "      <td>3.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4.000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3329 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "movieId  1      2      4      5      10     12     13     15     17     18     \\\n",
       "userId                                                                          \n",
       "0        4.000  5.000  3.000  4.000  5.000  4.000  2.000  4.000  2.000  2.000   \n",
       "1        0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000   \n",
       "2        0.000  0.000  0.000  3.000  4.000  3.000  0.000  5.000  0.000  4.000   \n",
       "3        0.000  0.000  0.000  0.000  1.000  0.000  0.000  0.000  0.000  0.000   \n",
       "4        0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000   \n",
       "\n",
       "movieId  ...    12612  12634  13346  14821  15721  15741  15826  16155  16323  \\\n",
       "userId   ...                                                                    \n",
       "0        ...    0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000   \n",
       "1        ...    0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000   \n",
       "2        ...    0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000   \n",
       "3        ...    0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000   \n",
       "4        ...    0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000   \n",
       "\n",
       "movieId  16660  \n",
       "userId          \n",
       "0        0.000  \n",
       "1        0.000  \n",
       "2        0.000  \n",
       "3        0.000  \n",
       "4        0.000  \n",
       "\n",
       "[5 rows x 3329 columns]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_pivot2 = rating_popular_movies.pivot(index='userId', columns='movieId',values='rating').fillna(0)\n",
    "ratings_pivot2_sparse = csr_matrix(ratings_pivot2.values)\n",
    "print(ratings_pivot2.shape)\n",
    "ratings_pivot2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3329, 27895)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X=ratings_pivot2.values.T\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3329, 10)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "svd=TruncatedSVD(n_components=10,random_state=17)\n",
    "matrix=svd.fit_transform(X)\n",
    "print(matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3329, 3329)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 1.        ,  0.88719504,  0.94722286, ...,  0.80781994,\n",
       "         0.7544022 ,  0.5269623 ],\n",
       "       [ 0.88719504,  1.        ,  0.86930487, ...,  0.61777898,\n",
       "         0.5339515 ,  0.67181041],\n",
       "       [ 0.94722286,  0.86930487,  1.        , ...,  0.79992334,\n",
       "         0.5870368 ,  0.45608533],\n",
       "       ..., \n",
       "       [ 0.80781994,  0.61777898,  0.79992334, ...,  1.        ,\n",
       "         0.74405944,  0.38598591],\n",
       "       [ 0.7544022 ,  0.5339515 ,  0.5870368 , ...,  0.74405944,\n",
       "         1.        ,  0.33090199],\n",
       "       [ 0.5269623 ,  0.67181041,  0.45608533, ...,  0.38598591,\n",
       "         0.33090199,  1.        ]])"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\",category=RuntimeWarning)\n",
    "corr=np.corrcoef(matrix)\n",
    "print(corr.shape)\n",
    "corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2550 3874 2552 3143 3732 2553  639 2547 2555 2956 2551]\n",
      "--------------------------------------------------------------\n",
      "[ 1.          0.99637533  0.99598866  0.99554685  0.99453251  0.99409538\n",
      "  0.99386483  0.99091663  0.98941127  0.98634359  0.98620206]\n"
     ]
    }
   ],
   "source": [
    "example_movieId=2550\n",
    "movieIds=ratings_pivot2.columns \n",
    "movieIds_list = list(movieIds)\n",
    "movieId_index = movieIds_list.index(example_movieId)\n",
    "\n",
    "movieId_vec=corr[movieId_index]\n",
    "argsort_idx =np.argsort(-movieId_vec)[:11]\n",
    "coff=movieId_vec[argsort_idx]\n",
    "similar_movie_Ids=movieIds[argsort_idx]\n",
    "print(similar_movie_Ids.values)\n",
    "print('--------------------------------------------------------------')\n",
    "print(coff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "当前电影: 黃飛鴻之三獅王爭霸\n",
      "推荐电影1: 太极张三丰, 相关系数:0.9963753317262949\n",
      "推荐电影2: 黃飛鴻之二男兒當自強, 相关系数:0.9959886617593083\n",
      "推荐电影3: 黃飛鴻, 相关系数:0.9955468545480034\n",
      "推荐电影4: 方世玉续集, 相关系数:0.9945325050301708\n",
      "推荐电影5: 新少林五祖, 相关系数:0.9940953835981864\n",
      "推荐电影6: 方世玉, 相关系数:0.9938648308354907\n",
      "推荐电影7: 倚天屠龍記之魔教教主, 相关系数:0.9909166322984471\n",
      "推荐电影8: 赌神, 相关系数:0.9894112727822967\n",
      "推荐电影9: 红番区, 相关系数:0.9863435881824448\n",
      "推荐电影10: 冒險王, 相关系数:0.9862020560988378\n"
     ]
    }
   ],
   "source": [
    "for idx,mId in enumerate(similar_movie_Ids):\n",
    "    name = movies[movies.movieId==mId]['title'].values[0]\n",
    "    if idx==0:\n",
    "        print('当前电影:',name)\n",
    "    else:\n",
    "         print('推荐电影{0}: {1}, 相关系数:{2}'.format(idx,name, coff[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
